<?php
/*
authored by Josh Fraser (www.joshfraser.com)
released under Apache License 2.0

Maintained by Alexander Makarov, http://rmcreative.ru/

$Id$
*/
error_reporting(E_ERROR|E_WARNING);
require_once("RollingCurl.php");
require_once("Util.php");
require_once("DbHelper.php");
//require_once('magpierss/rss_fetch.inc');

//<div class="realtime_list">[\s\S]*?</div>



$rc = new RollingCurl("request_callback");
$rc->request('http://www.cnbeta.com');
$rc->execute();
// $rss = fetch_rss($url);
// $fetchUrls = array();
// foreach ($rss->items as $item) {
// 	if (DbHelper::IsFetchUrl($item['link'])) {
//     	$fetchUrls[]= $item['link'];
//     }
// }
// single curl request

// $rc = new RollingCurl("content_callback");
// $rc->window_size = 20;
// foreach ($fetchUrls as $url) {
// 	$request = new RollingCurlRequest($url);
// 	$rc->add($request);
// }
// $rc->execute();

function request_callback($response, $info, $request)
{
	$urls=array();
	$dom = new DOMDocument();
	@$dom->loadHTML($response);
	$dom->normalize();
	$xpath = new DOMXPath($dom);
	///html/body/div/section[2]/section/section[3]/div[1]/div[2]/div/dl[1]/dt/a
	$realtime= $xpath->query('//*[@class="realtime_list"]/dl/dt/a/@href');
	$urls=array_merge($urls, getcontent($realtime)) ;
	$allnews = $xpath->query('//*[@id="allnews_all"]/div[1]/dl/dt/a/@href');
	$urls=array_merge($urls, getcontent($allnews)) ;

	$rc = new RollingCurl("content_callback");
	$rc->window_size = 20;
	foreach ($urls as $url) {
	  if (DbHelper::IsFetchUrl($url)) {
	 	$request = new RollingCurlRequest($url);
	 	$rc->add($request);
	  }
	}
	$rc->execute();
}

function getcontent($elements)
{
	$prefix = "http://www.cnbeta.com";
	$arr = array();
	if (!is_null($elements) ) {
		foreach ($elements as $element) {
		    $nodes = $element->childNodes;
		    foreach ($nodes as $node) {
		      $arr[]= $prefix.$node->nodeValue;
		    }
		  }
	}
	return $arr;
}




function content_callback($response, $info, $request)
{
	$regs= array(
				 'createtime'=>'<span class="date">(.*)?</span>',
				 'title'=>'<h2 id="news_title">(.*)?</h2>',
				 'content'=>'<div class="content">(<p [style|align]?.*>.*</p>)',
				 'summary' =>'<div class="introduction">[\s\S]*?(<p>[\s\S]*?)(?=</div>)',
				);	
	$flag= true;
	$data=array();
	foreach($regs as $key=>$value)
	{
		
		if (preg_match('~'.$value.'~i', $response, $out)) {
			//iconv("GB2312","UTF-8"."//IGNORE",$html);
			$content = $out[1];
			//$content=iconv("GB2312","UTF-8"."//IGNORE",$content);
			if($key=='createtime'){
				$content =strtotime($content);
				$today = strtotime(date("Y-m-d"));
				if($content<$today)
				{
					$flag=false;
					return;
				}
			}
			if($key=='content')
			{
				$imgs= Util::getimgs($content);
				foreach($imgs as $img){
					$content=str_replace($img['oldimg'],$img['newimg'],$content);
				}
				if (sizeof($imgs)>0) {
					$data['pic']=$imgs[0]['newimg'];
				}
			}
			if($key=='summary')
			{
				$content = strip_tags($content);
			}
			$data[$key]= $content;
		}
		else
		{
		    $flag=false;
			return;
		}
	}
	if($flag)
	{
		DbHelper::save($data,$request->url);
	}
}






// top 20 sites according to alexa (11/5/09)
/*
$urls = array("http://www.google.com",
              "http://www.facebook.com",
              "http://www.yahoo.com",
              "http://www.youtube.com",
              "http://www.live.com",
              "http://www.wikipedia.com",
              "http://www.blogger.com",
              "http://www.msn.com",
              "http://www.baidu.com",
              "http://www.yahoo.co.jp",
              "http://www.myspace.com",
              "http://www.qq.com",
              "http://www.google.co.in",
              "http://www.twitter.com",
              "http://www.google.de",
              "http://www.microsoft.com",
              "http://www.google.cn",
              "http://www.sina.com.cn",
              "http://www.wordpress.com",
              "http://www.google.co.uk");

$rc = new RollingCurl("request_callback");
$rc->window_size = 20;
foreach ($urls as $url) {
    $request = new RollingCurlRequest($url);
    $rc->add($request);
}
$rc->execute();
*/
